{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3_5_0_0\\lib\\site-packages\\h5py\\__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.layers import Dense,Dropout,Activation\n",
    "from hyperas.distributions import choice,uniform\n",
    "import numpy as np\n",
    "\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.utils import np_utils\n",
    "from hyperas import optim\n",
    "from hyperopt import tpe,Trials,STATUS_OK\n",
    "from keras.optimizers import RMSprop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getData():\n",
    "    '''\n",
    "    Data providing function:\n",
    "    This function is separated from model() so that hyperopt\n",
    "    won't reload data for each evaluation run.\n",
    "    '''\n",
    "    (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "    X_train = X_train.reshape(60000, 784)\n",
    "    X_test = X_test.reshape(10000, 784)\n",
    "    X_train = X_train.astype('float32')\n",
    "    X_test = X_test.astype('float32')\n",
    "    X_train /= 255\n",
    "    X_test /= 255\n",
    "    nb_classes = 10\n",
    "    Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
    "    Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
    "    return X_train, Y_train, X_test, Y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "??Activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model(X_train, Y_train, X_test, Y_test):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(512, input_shape=(784,)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout({{uniform(0, 1)}}))\n",
    "    model.add(Dense({{choice([256, 512, 1024])}}))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout({{uniform(0, 1)}}))\n",
    "    model.add(Dense(10))\n",
    "    model.add(Activation('softmax'))\n",
    "    rms = RMSprop()\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=rms, metrics=['accuracy'])\n",
    "    \n",
    "    model.fit(X_train, Y_train,\n",
    "              batch_size={{choice([64, 128])}},\n",
    "              epochs=10,\n",
    "              verbose=2,\n",
    "              validation_data=(X_test, Y_test))\n",
    "    score, acc = model.evaluate(X_test, Y_test, verbose=0)\n",
    "    print('Test accuracy:', acc)\n",
    "    return {'loss': -acc, 'status': STATUS_OK, 'model': model}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "??optim.minimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> Imports:\n",
      "#coding=utf-8\n",
      "\n",
      "try:\n",
      "    import keras\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.layers import Dense, Dropout, Activation\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from hyperas.distributions import choice, uniform\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    import numpy as np\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.datasets import mnist\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.models import Sequential\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.utils import np_utils\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from hyperas import optim\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from hyperopt import tpe, Trials, STATUS_OK\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.optimizers import RMSprop\n",
      "except:\n",
      "    pass\n",
      "\n",
      ">>> Hyperas search space:\n",
      "\n",
      "def get_space():\n",
      "    return {\n",
      "        'Dropout': hp.uniform('Dropout', 0, 1),\n",
      "        'Dense': hp.choice('Dense', [256, 512, 1024]),\n",
      "        'Dropout_1': hp.uniform('Dropout_1', 0, 1),\n",
      "        'batch_size': hp.choice('batch_size', [64, 128]),\n",
      "    }\n",
      "\n",
      ">>> Data\n",
      "  1: \n",
      "  2: '''\n",
      "  3: Data providing function:\n",
      "  4: This function is separated from model() so that hyperopt\n",
      "  5: won't reload data for each evaluation run.\n",
      "  6: '''\n",
      "  7: (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
      "  8: X_train = X_train.reshape(60000, 784)\n",
      "  9: X_test = X_test.reshape(10000, 784)\n",
      " 10: X_train = X_train.astype('float32')\n",
      " 11: X_test = X_test.astype('float32')\n",
      " 12: X_train /= 255\n",
      " 13: X_test /= 255\n",
      " 14: nb_classes = 10\n",
      " 15: Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
      " 16: Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
      " 17: \n",
      " 18: \n",
      " 19: \n",
      ">>> Resulting replaced keras model:\n",
      "\n",
      "  1: def keras_fmin_fnct(space):\n",
      "  2: \n",
      "  3:     model = Sequential()\n",
      "  4:     model.add(Dense(512, input_shape=(784,)))\n",
      "  5:     model.add(Activation('relu'))\n",
      "  6:     model.add(Dropout(space['Dropout']))\n",
      "  7:     model.add(Dense(space['Dense']))\n",
      "  8:     model.add(Activation('relu'))\n",
      "  9:     model.add(Dropout(space['Dropout_1']))\n",
      " 10:     model.add(Dense(10))\n",
      " 11:     model.add(Activation('softmax'))\n",
      " 12:     rms = RMSprop()\n",
      " 13:     model.compile(loss='categorical_crossentropy', optimizer=rms, metrics=['accuracy'])\n",
      " 14:     \n",
      " 15:     model.fit(X_train, Y_train,\n",
      " 16:               batch_size=space['batch_size'],\n",
      " 17:               epochs=10,\n",
      " 18:               verbose=2,\n",
      " 19:               validation_data=(X_test, Y_test))\n",
      " 20:     score, acc = model.evaluate(X_test, Y_test, verbose=0)\n",
      " 21:     print('Test accuracy:', acc)\n",
      " 22:     return {'loss': -acc, 'status': STATUS_OK, 'model': model}\n",
      " 23: \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\workspace\\kaggle\\python_dp_book\\temp_model.py:90: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      " - 5s - loss: 0.3686 - acc: 0.8890 - val_loss: 0.1443 - val_acc: 0.9590\n",
      "Test accuracy: 0.959\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      " - 5s - loss: 0.2599 - acc: 0.9218 - val_loss: 0.1184 - val_acc: 0.9639\n",
      "Test accuracy: 0.9639\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      " - 3s - loss: 0.5905 - acc: 0.8163 - val_loss: 0.1979 - val_acc: 0.9452\n",
      "Test accuracy: 0.9452\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      " - 4s - loss: 0.5809 - acc: 0.8220 - val_loss: 0.1882 - val_acc: 0.9465\n",
      "Test accuracy: 0.9465\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      " - 4s - loss: 0.4193 - acc: 0.8733 - val_loss: 0.1422 - val_acc: 0.9573\n",
      "Test accuracy: 0.9573\n",
      "Evalutation of best performing model:\n",
      "10000/10000 [==============================] - 0s 42us/step\n",
      "[0.11840711740292609, 0.9639]\n"
     ]
    }
   ],
   "source": [
    "X_train, Y_train, X_test, Y_test = getData()\n",
    "\n",
    "best_run, best_model,space = optim.minimize(model=model,\n",
    "                                      data=getData,\n",
    "                                      algo=tpe.suggest,\n",
    "                                      max_evals=5,\n",
    "                                      trials=Trials(),notebook_name=\"keras_hyperas_simple\",return_space=True)\n",
    "print(\"Evalutation of best performing model:\")\n",
    "print(best_model.evaluate(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Dense': 1, 'Dropout': 0.42522861686845626, 'Dropout_1': 0.23316134447477344, 'batch_size': 0}\n"
     ]
    }
   ],
   "source": [
    "print(best_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Dropout': <hyperopt.pyll.base.Apply object at 0x0000024662BDBA20>, 'Dense': <hyperopt.pyll.base.Apply object at 0x000002465F45A080>, 'Dropout_1': <hyperopt.pyll.base.Apply object at 0x0000024718D75E80>, 'batch_size': <hyperopt.pyll.base.Apply object at 0x0000024718D75F98>}\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
